{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 CartPoleをQ学習で制御"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 動画の描画関数の宣言\n",
    "# 参考URL http://nbviewer.jupyter.org/github/patrickmineault\n",
    "# /xcorr-notebooks/blob/master/Render%20OpenAI%20gym%20as%20GIF.ipynb\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "from matplotlib import animation\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    \"\"\"\n",
    "    Displays a list of frames as a gif, with controls\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(frames[0].shape[1]/72.0, frames[0].shape[0]/72.0),\n",
    "               dpi=72)\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "\n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames=len(frames),\n",
    "                                   interval=50)\n",
    "\n",
    "    anim.save('movie_cartpole.mp4')  # 動画のファイル名と保存です\n",
    "    display(display_animation(anim, default_mode='loop'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定数の設定\n",
    "ENV = 'CartPole-v0'  # 使用する課題名\n",
    "NUM_DIZITIZED = 6  # 各状態の離散値への分割数\n",
    "GAMMA = 0.99  # 時間割引率\n",
    "ETA = 0.5  # 学習係数\n",
    "MAX_STEPS = 200  # 1試行のstep数\n",
    "NUM_EPISODES = 1000  # 最大試行回数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    '''CartPoleのエージェントクラスです、棒付き台車そのものになります'''\n",
    "\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        self.brain = Brain(num_states, num_actions)  # エージェントが行動を決定するための頭脳を生成\n",
    "\n",
    "    def update_Q_function(self, observation, action, reward, observation_next):\n",
    "        '''Q関数の更新'''\n",
    "        self.brain.update_Q_table(\n",
    "            observation, action, reward, observation_next)\n",
    "\n",
    "    def get_action(self, observation, step):\n",
    "        '''行動の決定'''\n",
    "        action = self.brain.decide_action(observation, step)\n",
    "        return action\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Brain:\n",
    "    '''エージェントが持つ脳となるクラスです、Q学習を実行します'''\n",
    "\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        self.num_actions = num_actions  # CartPoleの行動（右に左に押す）の2を取得\n",
    "\n",
    "        # Qテーブルを作成。行数は状態を分割数^（4変数）にデジタル変換した値、列数は行動数を示す\n",
    "        self.q_table = np.random.uniform(low=0, high=1, size=(\n",
    "            NUM_DIZITIZED**num_states, num_actions))\n",
    "\n",
    "\n",
    "    def bins(self, clip_min, clip_max, num):\n",
    "        '''観測した状態（連続値）を離散値にデジタル変換する閾値を求める'''\n",
    "        return np.linspace(clip_min, clip_max, num + 1)[1:-1]\n",
    "\n",
    "    def digitize_state(self, observation):\n",
    "        '''観測したobservation状態を、離散値に変換する'''\n",
    "        cart_pos, cart_v, pole_angle, pole_v = observation\n",
    "        digitized = [\n",
    "            np.digitize(cart_pos, bins=self.bins(-2.4, 2.4, NUM_DIZITIZED)),\n",
    "            np.digitize(cart_v, bins=self.bins(-3.0, 3.0, NUM_DIZITIZED)),\n",
    "            np.digitize(pole_angle, bins=self.bins(-0.5, 0.5, NUM_DIZITIZED)),\n",
    "            np.digitize(pole_v, bins=self.bins(-2.0, 2.0, NUM_DIZITIZED))\n",
    "        ]\n",
    "        return sum([x * (NUM_DIZITIZED**i) for i, x in enumerate(digitized)])\n",
    "\n",
    "    def update_Q_table(self, observation, action, reward, observation_next):\n",
    "        '''QテーブルをQ学習により更新'''\n",
    "        state = self.digitize_state(observation)  # 状態を離散化\n",
    "        state_next = self.digitize_state(observation_next)  # 次の状態を離散化\n",
    "        Max_Q_next = max(self.q_table[state_next][:])\n",
    "        self.q_table[state, action] = self.q_table[state, action] + \\\n",
    "            ETA * (reward + GAMMA * Max_Q_next - self.q_table[state, action])\n",
    "\n",
    "    def decide_action(self, observation, episode):\n",
    "        '''ε-greedy法で徐々に最適行動のみを採用する'''\n",
    "        state = self.digitize_state(observation)\n",
    "        epsilon = 0.5 * (1 / (episode + 1))\n",
    "\n",
    "        if epsilon <= np.random.uniform(0, 1):\n",
    "            action = np.argmax(self.q_table[state][:])\n",
    "        else:\n",
    "            action = np.random.choice(self.num_actions)  # 0,1の行動をランダムに返す\n",
    "        return action\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Environment:\n",
    "    '''CartPoleを実行する環境のクラスです'''\n",
    "\n",
    "    def __init__(self):\n",
    "        self.env = gym.make(ENV)  # 実行する課題を設定\n",
    "        num_states = self.env.observation_space.shape[0]  # 課題の状態の数4を取得\n",
    "        num_actions = self.env.action_space.n  # CartPoleの行動（右に左に押す）の2を取得\n",
    "        self.agent = Agent(num_states, num_actions)  # 環境内で行動するAgentを生成\n",
    "\n",
    "    def run(self):\n",
    "        '''実行'''\n",
    "        complete_episodes = 0  # 195step以上連続で立ち続けた試行数\n",
    "        is_episode_final = False  # 最終試行フラグ\n",
    "        frames = []  # 動画用に画像を格納する変数\n",
    "\n",
    "        for episode in range(NUM_EPISODES):  # 試行数分繰り返す\n",
    "            observation = self.env.reset()  # 環境の初期化\n",
    "\n",
    "            for step in range(MAX_STEPS):  # 1エピソードのループ\n",
    "\n",
    "                if is_episode_final is True:  # 最終試行ではframesに各時刻の画像を追加していく\n",
    "                    frames.append(self.env.render(mode='rgb_array'))\n",
    "\n",
    "                # 行動を求める\n",
    "                action = self.agent.get_action(observation, episode)\n",
    "\n",
    "                # 行動a_tの実行により、s_{t+1}, r_{t+1}を求める\n",
    "                observation_next, _, done, _ = self.env.step(\n",
    "                    action)  # rewardとinfoは使わないので_にする\n",
    "\n",
    "                # 報酬を与える\n",
    "                if done:  # ステップ数が200経過するか、一定角度以上傾くとdoneはtrueになる\n",
    "                    if step < 195:\n",
    "                        reward = -1  # 途中でこけたら罰則として報酬-1を与える\n",
    "                        complete_episodes = 0  # 195step以上連続で立ち続けた試行数をリセット\n",
    "                    else:\n",
    "                        reward = 1  # 立ったまま終了時は報酬1を与える\n",
    "                        complete_episodes += 1  # 連続記録を更新\n",
    "                else:\n",
    "                    reward = 0  # 途中の報酬は0\n",
    "\n",
    "                # step+1の状態observation_nextを用いて,Q関数を更新する\n",
    "                self.agent.update_Q_function(\n",
    "                    observation, action, reward, observation_next)\n",
    "\n",
    "                # 観測の更新\n",
    "                observation = observation_next\n",
    "\n",
    "                # 終了時の処理\n",
    "                if done:\n",
    "                    print('{0} Episode: Finished after {1} time steps'.format(\n",
    "                        episode, step + 1))\n",
    "                    break\n",
    "\n",
    "            if is_episode_final is True:  # 最終試行では動画を保存と描画\n",
    "                display_frames_as_gif(frames)\n",
    "                break\n",
    "\n",
    "            if complete_episodes >= 10:  # 10連続成功なら\n",
    "                print('10回連続成功')\n",
    "                is_episode_final = True  # 次の試行を描画を行う最終試行とする\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "0 Episode: Finished after 22 time steps\n",
      "1 Episode: Finished after 27 time steps\n",
      "2 Episode: Finished after 12 time steps\n",
      "3 Episode: Finished after 17 time steps\n",
      "4 Episode: Finished after 15 time steps\n",
      "5 Episode: Finished after 10 time steps\n",
      "6 Episode: Finished after 15 time steps\n",
      "7 Episode: Finished after 8 time steps\n",
      "8 Episode: Finished after 23 time steps\n",
      "9 Episode: Finished after 43 time steps\n",
      "10 Episode: Finished after 11 time steps\n",
      "11 Episode: Finished after 20 time steps\n",
      "12 Episode: Finished after 12 time steps\n",
      "13 Episode: Finished after 16 time steps\n",
      "14 Episode: Finished after 14 time steps\n",
      "15 Episode: Finished after 10 time steps\n",
      "16 Episode: Finished after 18 time steps\n",
      "17 Episode: Finished after 12 time steps\n",
      "18 Episode: Finished after 11 time steps\n",
      "19 Episode: Finished after 31 time steps\n",
      "20 Episode: Finished after 13 time steps\n",
      "21 Episode: Finished after 13 time steps\n",
      "22 Episode: Finished after 34 time steps\n",
      "23 Episode: Finished after 11 time steps\n",
      "24 Episode: Finished after 8 time steps\n",
      "25 Episode: Finished after 25 time steps\n",
      "26 Episode: Finished after 16 time steps\n",
      "27 Episode: Finished after 12 time steps\n",
      "28 Episode: Finished after 23 time steps\n",
      "29 Episode: Finished after 17 time steps\n",
      "30 Episode: Finished after 13 time steps\n",
      "31 Episode: Finished after 78 time steps\n",
      "32 Episode: Finished after 17 time steps\n",
      "33 Episode: Finished after 15 time steps\n",
      "34 Episode: Finished after 141 time steps\n",
      "35 Episode: Finished after 50 time steps\n",
      "36 Episode: Finished after 61 time steps\n",
      "37 Episode: Finished after 192 time steps\n",
      "38 Episode: Finished after 200 time steps\n",
      "39 Episode: Finished after 22 time steps\n",
      "40 Episode: Finished after 165 time steps\n",
      "41 Episode: Finished after 21 time steps\n",
      "42 Episode: Finished after 15 time steps\n",
      "43 Episode: Finished after 65 time steps\n",
      "44 Episode: Finished after 102 time steps\n",
      "45 Episode: Finished after 49 time steps\n",
      "46 Episode: Finished after 86 time steps\n",
      "47 Episode: Finished after 160 time steps\n",
      "48 Episode: Finished after 19 time steps\n",
      "49 Episode: Finished after 21 time steps\n",
      "50 Episode: Finished after 26 time steps\n",
      "51 Episode: Finished after 26 time steps\n",
      "52 Episode: Finished after 115 time steps\n",
      "53 Episode: Finished after 63 time steps\n",
      "54 Episode: Finished after 82 time steps\n",
      "55 Episode: Finished after 23 time steps\n",
      "56 Episode: Finished after 24 time steps\n",
      "57 Episode: Finished after 27 time steps\n",
      "58 Episode: Finished after 140 time steps\n",
      "59 Episode: Finished after 123 time steps\n",
      "60 Episode: Finished after 34 time steps\n",
      "61 Episode: Finished after 63 time steps\n",
      "62 Episode: Finished after 176 time steps\n",
      "63 Episode: Finished after 200 time steps\n",
      "64 Episode: Finished after 70 time steps\n",
      "65 Episode: Finished after 14 time steps\n",
      "66 Episode: Finished after 156 time steps\n",
      "67 Episode: Finished after 183 time steps\n",
      "68 Episode: Finished after 129 time steps\n",
      "69 Episode: Finished after 16 time steps\n",
      "70 Episode: Finished after 9 time steps\n",
      "71 Episode: Finished after 43 time steps\n",
      "72 Episode: Finished after 22 time steps\n",
      "73 Episode: Finished after 108 time steps\n",
      "74 Episode: Finished after 15 time steps\n",
      "75 Episode: Finished after 60 time steps\n",
      "76 Episode: Finished after 10 time steps\n",
      "77 Episode: Finished after 71 time steps\n",
      "78 Episode: Finished after 26 time steps\n",
      "79 Episode: Finished after 45 time steps\n",
      "80 Episode: Finished after 32 time steps\n",
      "81 Episode: Finished after 60 time steps\n",
      "82 Episode: Finished after 65 time steps\n",
      "83 Episode: Finished after 88 time steps\n",
      "84 Episode: Finished after 67 time steps\n",
      "85 Episode: Finished after 67 time steps\n",
      "86 Episode: Finished after 46 time steps\n",
      "87 Episode: Finished after 69 time steps\n",
      "88 Episode: Finished after 31 time steps\n",
      "89 Episode: Finished after 200 time steps\n",
      "90 Episode: Finished after 57 time steps\n",
      "91 Episode: Finished after 200 time steps\n",
      "92 Episode: Finished after 138 time steps\n",
      "93 Episode: Finished after 131 time steps\n",
      "94 Episode: Finished after 98 time steps\n",
      "95 Episode: Finished after 112 time steps\n",
      "96 Episode: Finished after 146 time steps\n",
      "97 Episode: Finished after 200 time steps\n",
      "98 Episode: Finished after 200 time steps\n",
      "99 Episode: Finished after 129 time steps\n",
      "100 Episode: Finished after 124 time steps\n",
      "101 Episode: Finished after 174 time steps\n",
      "102 Episode: Finished after 200 time steps\n",
      "103 Episode: Finished after 200 time steps\n",
      "104 Episode: Finished after 200 time steps\n",
      "105 Episode: Finished after 69 time steps\n",
      "106 Episode: Finished after 200 time steps\n",
      "107 Episode: Finished after 200 time steps\n",
      "108 Episode: Finished after 108 time steps\n",
      "109 Episode: Finished after 16 time steps\n",
      "110 Episode: Finished after 23 time steps\n",
      "111 Episode: Finished after 115 time steps\n",
      "112 Episode: Finished after 45 time steps\n",
      "113 Episode: Finished after 85 time steps\n",
      "114 Episode: Finished after 174 time steps\n",
      "115 Episode: Finished after 28 time steps\n",
      "116 Episode: Finished after 73 time steps\n",
      "117 Episode: Finished after 40 time steps\n",
      "118 Episode: Finished after 35 time steps\n",
      "119 Episode: Finished after 45 time steps\n",
      "120 Episode: Finished after 158 time steps\n",
      "121 Episode: Finished after 67 time steps\n",
      "122 Episode: Finished after 148 time steps\n",
      "123 Episode: Finished after 95 time steps\n",
      "124 Episode: Finished after 33 time steps\n",
      "125 Episode: Finished after 134 time steps\n",
      "126 Episode: Finished after 52 time steps\n",
      "127 Episode: Finished after 197 time steps\n",
      "128 Episode: Finished after 150 time steps\n",
      "129 Episode: Finished after 200 time steps\n",
      "130 Episode: Finished after 150 time steps\n",
      "131 Episode: Finished after 96 time steps\n",
      "132 Episode: Finished after 31 time steps\n",
      "133 Episode: Finished after 200 time steps\n",
      "134 Episode: Finished after 185 time steps\n",
      "135 Episode: Finished after 142 time steps\n",
      "136 Episode: Finished after 165 time steps\n",
      "137 Episode: Finished after 200 time steps\n",
      "138 Episode: Finished after 197 time steps\n",
      "139 Episode: Finished after 150 time steps\n",
      "140 Episode: Finished after 50 time steps\n",
      "141 Episode: Finished after 200 time steps\n",
      "142 Episode: Finished after 200 time steps\n",
      "143 Episode: Finished after 133 time steps\n",
      "144 Episode: Finished after 156 time steps\n",
      "145 Episode: Finished after 101 time steps\n",
      "146 Episode: Finished after 176 time steps\n",
      "147 Episode: Finished after 184 time steps\n",
      "148 Episode: Finished after 196 time steps\n",
      "149 Episode: Finished after 166 time steps\n",
      "150 Episode: Finished after 98 time steps\n",
      "151 Episode: Finished after 200 time steps\n",
      "152 Episode: Finished after 33 time steps\n",
      "153 Episode: Finished after 117 time steps\n",
      "154 Episode: Finished after 81 time steps\n",
      "155 Episode: Finished after 139 time steps\n",
      "156 Episode: Finished after 200 time steps\n",
      "157 Episode: Finished after 178 time steps\n",
      "158 Episode: Finished after 22 time steps\n",
      "159 Episode: Finished after 200 time steps\n",
      "160 Episode: Finished after 200 time steps\n",
      "161 Episode: Finished after 138 time steps\n",
      "162 Episode: Finished after 32 time steps\n",
      "163 Episode: Finished after 144 time steps\n",
      "164 Episode: Finished after 200 time steps\n",
      "165 Episode: Finished after 200 time steps\n",
      "166 Episode: Finished after 200 time steps\n",
      "167 Episode: Finished after 200 time steps\n",
      "168 Episode: Finished after 200 time steps\n",
      "169 Episode: Finished after 200 time steps\n",
      "170 Episode: Finished after 200 time steps\n",
      "171 Episode: Finished after 200 time steps\n",
      "172 Episode: Finished after 200 time steps\n",
      "173 Episode: Finished after 120 time steps\n",
      "174 Episode: Finished after 200 time steps\n",
      "175 Episode: Finished after 200 time steps\n",
      "176 Episode: Finished after 133 time steps\n",
      "177 Episode: Finished after 183 time steps\n",
      "178 Episode: Finished after 200 time steps\n",
      "179 Episode: Finished after 168 time steps\n",
      "180 Episode: Finished after 171 time steps\n",
      "181 Episode: Finished after 200 time steps\n",
      "182 Episode: Finished after 120 time steps\n",
      "183 Episode: Finished after 105 time steps\n",
      "184 Episode: Finished after 181 time steps\n",
      "185 Episode: Finished after 105 time steps\n",
      "186 Episode: Finished after 178 time steps\n",
      "187 Episode: Finished after 137 time steps\n",
      "188 Episode: Finished after 94 time steps\n",
      "189 Episode: Finished after 153 time steps\n",
      "190 Episode: Finished after 107 time steps\n",
      "191 Episode: Finished after 120 time steps\n",
      "192 Episode: Finished after 122 time steps\n",
      "193 Episode: Finished after 125 time steps\n",
      "194 Episode: Finished after 148 time steps\n",
      "195 Episode: Finished after 200 time steps\n",
      "196 Episode: Finished after 86 time steps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "197 Episode: Finished after 132 time steps\n",
      "198 Episode: Finished after 139 time steps\n",
      "199 Episode: Finished after 200 time steps\n",
      "200 Episode: Finished after 163 time steps\n",
      "201 Episode: Finished after 168 time steps\n",
      "202 Episode: Finished after 200 time steps\n",
      "203 Episode: Finished after 200 time steps\n",
      "204 Episode: Finished after 166 time steps\n",
      "205 Episode: Finished after 200 time steps\n",
      "206 Episode: Finished after 200 time steps\n",
      "207 Episode: Finished after 191 time steps\n",
      "208 Episode: Finished after 101 time steps\n",
      "209 Episode: Finished after 200 time steps\n",
      "210 Episode: Finished after 100 time steps\n",
      "211 Episode: Finished after 159 time steps\n",
      "212 Episode: Finished after 200 time steps\n",
      "213 Episode: Finished after 144 time steps\n",
      "214 Episode: Finished after 200 time steps\n",
      "215 Episode: Finished after 200 time steps\n",
      "216 Episode: Finished after 172 time steps\n",
      "217 Episode: Finished after 194 time steps\n",
      "218 Episode: Finished after 200 time steps\n",
      "219 Episode: Finished after 172 time steps\n",
      "220 Episode: Finished after 193 time steps\n",
      "221 Episode: Finished after 193 time steps\n",
      "222 Episode: Finished after 200 time steps\n",
      "223 Episode: Finished after 151 time steps\n",
      "224 Episode: Finished after 200 time steps\n",
      "225 Episode: Finished after 125 time steps\n",
      "226 Episode: Finished after 149 time steps\n",
      "227 Episode: Finished after 200 time steps\n",
      "228 Episode: Finished after 187 time steps\n",
      "229 Episode: Finished after 69 time steps\n",
      "230 Episode: Finished after 151 time steps\n",
      "231 Episode: Finished after 77 time steps\n",
      "232 Episode: Finished after 141 time steps\n",
      "233 Episode: Finished after 200 time steps\n",
      "234 Episode: Finished after 139 time steps\n",
      "235 Episode: Finished after 200 time steps\n",
      "236 Episode: Finished after 200 time steps\n",
      "237 Episode: Finished after 136 time steps\n",
      "238 Episode: Finished after 145 time steps\n",
      "239 Episode: Finished after 48 time steps\n",
      "240 Episode: Finished after 86 time steps\n",
      "241 Episode: Finished after 174 time steps\n",
      "242 Episode: Finished after 127 time steps\n",
      "243 Episode: Finished after 200 time steps\n",
      "244 Episode: Finished after 56 time steps\n",
      "245 Episode: Finished after 148 time steps\n",
      "246 Episode: Finished after 160 time steps\n",
      "247 Episode: Finished after 57 time steps\n",
      "248 Episode: Finished after 190 time steps\n",
      "249 Episode: Finished after 133 time steps\n",
      "250 Episode: Finished after 110 time steps\n",
      "251 Episode: Finished after 119 time steps\n",
      "252 Episode: Finished after 133 time steps\n",
      "253 Episode: Finished after 115 time steps\n",
      "254 Episode: Finished after 99 time steps\n",
      "255 Episode: Finished after 68 time steps\n",
      "256 Episode: Finished after 35 time steps\n",
      "257 Episode: Finished after 29 time steps\n",
      "258 Episode: Finished after 200 time steps\n",
      "259 Episode: Finished after 158 time steps\n",
      "260 Episode: Finished after 200 time steps\n",
      "261 Episode: Finished after 195 time steps\n",
      "262 Episode: Finished after 200 time steps\n",
      "263 Episode: Finished after 96 time steps\n",
      "264 Episode: Finished after 146 time steps\n",
      "265 Episode: Finished after 137 time steps\n",
      "266 Episode: Finished after 139 time steps\n",
      "267 Episode: Finished after 103 time steps\n",
      "268 Episode: Finished after 175 time steps\n",
      "269 Episode: Finished after 200 time steps\n",
      "270 Episode: Finished after 162 time steps\n",
      "271 Episode: Finished after 139 time steps\n",
      "272 Episode: Finished after 200 time steps\n",
      "273 Episode: Finished after 200 time steps\n",
      "274 Episode: Finished after 190 time steps\n",
      "275 Episode: Finished after 117 time steps\n",
      "276 Episode: Finished after 114 time steps\n",
      "277 Episode: Finished after 40 time steps\n",
      "278 Episode: Finished after 19 time steps\n",
      "279 Episode: Finished after 167 time steps\n",
      "280 Episode: Finished after 125 time steps\n",
      "281 Episode: Finished after 119 time steps\n",
      "282 Episode: Finished after 95 time steps\n",
      "283 Episode: Finished after 200 time steps\n",
      "284 Episode: Finished after 181 time steps\n",
      "285 Episode: Finished after 29 time steps\n",
      "286 Episode: Finished after 165 time steps\n",
      "287 Episode: Finished after 192 time steps\n",
      "288 Episode: Finished after 200 time steps\n",
      "289 Episode: Finished after 137 time steps\n",
      "290 Episode: Finished after 200 time steps\n",
      "291 Episode: Finished after 84 time steps\n",
      "292 Episode: Finished after 200 time steps\n",
      "293 Episode: Finished after 157 time steps\n",
      "294 Episode: Finished after 196 time steps\n",
      "295 Episode: Finished after 200 time steps\n",
      "296 Episode: Finished after 29 time steps\n",
      "297 Episode: Finished after 200 time steps\n",
      "298 Episode: Finished after 200 time steps\n",
      "299 Episode: Finished after 200 time steps\n",
      "300 Episode: Finished after 200 time steps\n",
      "301 Episode: Finished after 200 time steps\n",
      "302 Episode: Finished after 200 time steps\n",
      "303 Episode: Finished after 172 time steps\n",
      "304 Episode: Finished after 151 time steps\n",
      "305 Episode: Finished after 35 time steps\n",
      "306 Episode: Finished after 58 time steps\n",
      "307 Episode: Finished after 27 time steps\n",
      "308 Episode: Finished after 8 time steps\n",
      "309 Episode: Finished after 37 time steps\n",
      "310 Episode: Finished after 35 time steps\n",
      "311 Episode: Finished after 200 time steps\n",
      "312 Episode: Finished after 181 time steps\n",
      "313 Episode: Finished after 200 time steps\n",
      "314 Episode: Finished after 200 time steps\n",
      "315 Episode: Finished after 200 time steps\n",
      "316 Episode: Finished after 200 time steps\n",
      "317 Episode: Finished after 24 time steps\n",
      "318 Episode: Finished after 200 time steps\n",
      "319 Episode: Finished after 200 time steps\n",
      "320 Episode: Finished after 200 time steps\n",
      "321 Episode: Finished after 200 time steps\n",
      "322 Episode: Finished after 200 time steps\n",
      "323 Episode: Finished after 200 time steps\n",
      "324 Episode: Finished after 200 time steps\n",
      "325 Episode: Finished after 200 time steps\n",
      "326 Episode: Finished after 57 time steps\n",
      "327 Episode: Finished after 162 time steps\n",
      "328 Episode: Finished after 199 time steps\n",
      "329 Episode: Finished after 71 time steps\n",
      "330 Episode: Finished after 200 time steps\n",
      "331 Episode: Finished after 200 time steps\n",
      "332 Episode: Finished after 156 time steps\n",
      "333 Episode: Finished after 61 time steps\n",
      "334 Episode: Finished after 196 time steps\n",
      "335 Episode: Finished after 200 time steps\n",
      "336 Episode: Finished after 200 time steps\n",
      "337 Episode: Finished after 194 time steps\n",
      "338 Episode: Finished after 200 time steps\n",
      "339 Episode: Finished after 73 time steps\n",
      "340 Episode: Finished after 200 time steps\n",
      "341 Episode: Finished after 56 time steps\n",
      "342 Episode: Finished after 200 time steps\n",
      "343 Episode: Finished after 200 time steps\n",
      "344 Episode: Finished after 187 time steps\n",
      "345 Episode: Finished after 200 time steps\n",
      "346 Episode: Finished after 29 time steps\n",
      "347 Episode: Finished after 200 time steps\n",
      "348 Episode: Finished after 200 time steps\n",
      "349 Episode: Finished after 200 time steps\n",
      "350 Episode: Finished after 200 time steps\n",
      "351 Episode: Finished after 200 time steps\n",
      "352 Episode: Finished after 200 time steps\n",
      "353 Episode: Finished after 200 time steps\n",
      "354 Episode: Finished after 200 time steps\n",
      "355 Episode: Finished after 93 time steps\n",
      "356 Episode: Finished after 200 time steps\n",
      "357 Episode: Finished after 200 time steps\n",
      "358 Episode: Finished after 191 time steps\n",
      "359 Episode: Finished after 69 time steps\n",
      "360 Episode: Finished after 182 time steps\n",
      "361 Episode: Finished after 36 time steps\n",
      "362 Episode: Finished after 200 time steps\n",
      "363 Episode: Finished after 200 time steps\n",
      "364 Episode: Finished after 192 time steps\n",
      "365 Episode: Finished after 198 time steps\n",
      "366 Episode: Finished after 200 time steps\n",
      "367 Episode: Finished after 200 time steps\n",
      "368 Episode: Finished after 63 time steps\n",
      "369 Episode: Finished after 200 time steps\n",
      "370 Episode: Finished after 164 time steps\n",
      "371 Episode: Finished after 108 time steps\n",
      "372 Episode: Finished after 77 time steps\n",
      "373 Episode: Finished after 200 time steps\n",
      "374 Episode: Finished after 200 time steps\n",
      "375 Episode: Finished after 200 time steps\n",
      "376 Episode: Finished after 161 time steps\n",
      "377 Episode: Finished after 200 time steps\n",
      "378 Episode: Finished after 200 time steps\n",
      "379 Episode: Finished after 89 time steps\n",
      "380 Episode: Finished after 195 time steps\n",
      "381 Episode: Finished after 200 time steps\n",
      "382 Episode: Finished after 200 time steps\n",
      "383 Episode: Finished after 45 time steps\n",
      "384 Episode: Finished after 200 time steps\n",
      "385 Episode: Finished after 101 time steps\n",
      "386 Episode: Finished after 200 time steps\n",
      "387 Episode: Finished after 200 time steps\n",
      "388 Episode: Finished after 200 time steps\n",
      "389 Episode: Finished after 36 time steps\n",
      "390 Episode: Finished after 102 time steps\n",
      "391 Episode: Finished after 200 time steps\n",
      "392 Episode: Finished after 200 time steps\n",
      "393 Episode: Finished after 200 time steps\n",
      "394 Episode: Finished after 130 time steps\n",
      "395 Episode: Finished after 113 time steps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "396 Episode: Finished after 154 time steps\n",
      "397 Episode: Finished after 193 time steps\n",
      "398 Episode: Finished after 200 time steps\n",
      "399 Episode: Finished after 177 time steps\n",
      "400 Episode: Finished after 118 time steps\n",
      "401 Episode: Finished after 168 time steps\n",
      "402 Episode: Finished after 200 time steps\n",
      "403 Episode: Finished after 151 time steps\n",
      "404 Episode: Finished after 200 time steps\n",
      "405 Episode: Finished after 105 time steps\n",
      "406 Episode: Finished after 200 time steps\n",
      "407 Episode: Finished after 38 time steps\n",
      "408 Episode: Finished after 149 time steps\n",
      "409 Episode: Finished after 200 time steps\n",
      "410 Episode: Finished after 110 time steps\n",
      "411 Episode: Finished after 198 time steps\n",
      "412 Episode: Finished after 200 time steps\n",
      "413 Episode: Finished after 198 time steps\n",
      "414 Episode: Finished after 197 time steps\n",
      "415 Episode: Finished after 200 time steps\n",
      "416 Episode: Finished after 193 time steps\n",
      "417 Episode: Finished after 200 time steps\n",
      "418 Episode: Finished after 200 time steps\n",
      "419 Episode: Finished after 134 time steps\n",
      "420 Episode: Finished after 200 time steps\n",
      "421 Episode: Finished after 200 time steps\n",
      "422 Episode: Finished after 156 time steps\n",
      "423 Episode: Finished after 157 time steps\n",
      "424 Episode: Finished after 200 time steps\n",
      "425 Episode: Finished after 33 time steps\n",
      "426 Episode: Finished after 200 time steps\n",
      "427 Episode: Finished after 200 time steps\n",
      "428 Episode: Finished after 22 time steps\n",
      "429 Episode: Finished after 176 time steps\n",
      "430 Episode: Finished after 200 time steps\n",
      "431 Episode: Finished after 200 time steps\n",
      "432 Episode: Finished after 168 time steps\n",
      "433 Episode: Finished after 187 time steps\n",
      "434 Episode: Finished after 200 time steps\n",
      "435 Episode: Finished after 107 time steps\n",
      "436 Episode: Finished after 111 time steps\n",
      "437 Episode: Finished after 200 time steps\n",
      "438 Episode: Finished after 200 time steps\n",
      "439 Episode: Finished after 185 time steps\n",
      "440 Episode: Finished after 200 time steps\n",
      "441 Episode: Finished after 200 time steps\n",
      "442 Episode: Finished after 138 time steps\n",
      "443 Episode: Finished after 200 time steps\n",
      "444 Episode: Finished after 163 time steps\n",
      "445 Episode: Finished after 200 time steps\n",
      "446 Episode: Finished after 200 time steps\n",
      "447 Episode: Finished after 200 time steps\n",
      "448 Episode: Finished after 160 time steps\n",
      "449 Episode: Finished after 188 time steps\n",
      "450 Episode: Finished after 103 time steps\n",
      "451 Episode: Finished after 200 time steps\n",
      "452 Episode: Finished after 200 time steps\n",
      "453 Episode: Finished after 79 time steps\n",
      "454 Episode: Finished after 200 time steps\n",
      "455 Episode: Finished after 22 time steps\n",
      "456 Episode: Finished after 200 time steps\n",
      "457 Episode: Finished after 200 time steps\n",
      "458 Episode: Finished after 200 time steps\n",
      "459 Episode: Finished after 200 time steps\n",
      "460 Episode: Finished after 200 time steps\n",
      "461 Episode: Finished after 193 time steps\n",
      "462 Episode: Finished after 200 time steps\n",
      "463 Episode: Finished after 200 time steps\n",
      "464 Episode: Finished after 200 time steps\n",
      "465 Episode: Finished after 200 time steps\n",
      "466 Episode: Finished after 200 time steps\n",
      "467 Episode: Finished after 200 time steps\n",
      "468 Episode: Finished after 200 time steps\n",
      "469 Episode: Finished after 177 time steps\n",
      "470 Episode: Finished after 200 time steps\n",
      "471 Episode: Finished after 185 time steps\n",
      "472 Episode: Finished after 200 time steps\n",
      "473 Episode: Finished after 200 time steps\n",
      "474 Episode: Finished after 131 time steps\n",
      "475 Episode: Finished after 200 time steps\n",
      "476 Episode: Finished after 200 time steps\n",
      "477 Episode: Finished after 200 time steps\n",
      "478 Episode: Finished after 200 time steps\n",
      "479 Episode: Finished after 200 time steps\n",
      "480 Episode: Finished after 200 time steps\n",
      "481 Episode: Finished after 200 time steps\n",
      "482 Episode: Finished after 169 time steps\n",
      "483 Episode: Finished after 188 time steps\n",
      "484 Episode: Finished after 200 time steps\n",
      "485 Episode: Finished after 200 time steps\n",
      "486 Episode: Finished after 200 time steps\n",
      "487 Episode: Finished after 172 time steps\n",
      "488 Episode: Finished after 167 time steps\n",
      "489 Episode: Finished after 200 time steps\n",
      "490 Episode: Finished after 200 time steps\n",
      "491 Episode: Finished after 200 time steps\n",
      "492 Episode: Finished after 200 time steps\n",
      "493 Episode: Finished after 200 time steps\n",
      "494 Episode: Finished after 200 time steps\n",
      "495 Episode: Finished after 200 time steps\n",
      "496 Episode: Finished after 200 time steps\n",
      "497 Episode: Finished after 200 time steps\n",
      "498 Episode: Finished after 200 time steps\n",
      "10回連続成功\n",
      "499 Episode: Finished after 200 time steps\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Define the Animation class */\n",
       "  function Animation(frames, img_id, slider_id, interval, loop_select_id){\n",
       "    this.img_id = img_id;\n",
       "    this.slider_id = slider_id;\n",
       "    this.loop_select_id = loop_select_id;\n",
       "    this.interval = interval;\n",
       "    this.current_frame = 0;\n",
       "    this.direction = 0;\n",
       "    this.timer = null;\n",
       "    this.frames = new Array(frames.length);\n",
       "\n",
       "    for (var i=0; i<frames.length; i++)\n",
       "    {\n",
       "     this.frames[i] = new Image();\n",
       "     this.frames[i].src = frames[i];\n",
       "    }\n",
       "    document.getElementById(this.slider_id).max = this.frames.length - 1;\n",
       "    this.set_frame(this.current_frame);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.get_loop_state = function(){\n",
       "    var button_group = document[this.loop_select_id].state;\n",
       "    for (var i = 0; i < button_group.length; i++) {\n",
       "        var button = button_group[i];\n",
       "        if (button.checked) {\n",
       "            return button.value;\n",
       "        }\n",
       "    }\n",
       "    return undefined;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.set_frame = function(frame){\n",
       "    this.current_frame = frame;\n",
       "    document.getElementById(this.img_id).src = this.frames[this.current_frame].src;\n",
       "    document.getElementById(this.slider_id).value = this.current_frame;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.next_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.min(this.frames.length - 1, this.current_frame + 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.previous_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.max(0, this.current_frame - 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.first_frame = function()\n",
       "  {\n",
       "    this.set_frame(0);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.last_frame = function()\n",
       "  {\n",
       "    this.set_frame(this.frames.length - 1);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.slower = function()\n",
       "  {\n",
       "    this.interval /= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.faster = function()\n",
       "  {\n",
       "    this.interval *= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_forward = function()\n",
       "  {\n",
       "    this.current_frame += 1;\n",
       "    if(this.current_frame < this.frames.length){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.first_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.last_frame();\n",
       "        this.reverse_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.last_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_reverse = function()\n",
       "  {\n",
       "    this.current_frame -= 1;\n",
       "    if(this.current_frame >= 0){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.last_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.first_frame();\n",
       "        this.play_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.first_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.pause_animation = function()\n",
       "  {\n",
       "    this.direction = 0;\n",
       "    if (this.timer){\n",
       "      clearInterval(this.timer);\n",
       "      this.timer = null;\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.play_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = 1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_forward();}, this.interval);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.reverse_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = -1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_reverse();}, this.interval);\n",
       "  }\n",
       "</script>\n",
       "\n",
       "<div class=\"animation\" align=\"center\">\n",
       "    <img id=\"_anim_imgTGHEMQVWAJRGKPBH\">\n",
       "    <br>\n",
       "    <input id=\"_anim_sliderTGHEMQVWAJRGKPBH\" type=\"range\" style=\"width:350px\" name=\"points\" min=\"0\" max=\"1\" step=\"1\" value=\"0\" onchange=\"animTGHEMQVWAJRGKPBH.set_frame(parseInt(this.value));\"></input>\n",
       "    <br>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.slower()\">&#8211;</button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.first_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.previous_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.reverse_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.pause_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.play_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.next_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.last_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animTGHEMQVWAJRGKPBH.faster()\">+</button>\n",
       "  <form action=\"#n\" name=\"_anim_loop_selectTGHEMQVWAJRGKPBH\" class=\"anim_control\">\n",
       "    <input type=\"radio\" name=\"state\" value=\"once\" > Once </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"loop\" checked> Loop </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"reflect\" > Reflect </input>\n",
       "  </form>\n",
       "</div>\n",
       "\n",
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Instantiate the Animation class. */\n",
       "  /* The IDs given should match those used in the template above. */\n",
       "  (function() {\n",
       "    var img_id = \"_anim_imgTGHEMQVWAJRGKPBH\";\n",
       "    var slider_id = \"_anim_sliderTGHEMQVWAJRGKPBH\";\n",
       "    var loop_select_id = \"_anim_loop_selectTGHEMQVWAJRGKPBH\";\n",
       "    var frames = new Array(0);\n",
       "    \n",
       "  frames[0] = \"\"\n",
       "  frames[1] = \"\"\n",
       "  frames[2] = \"\"\n",
       "  frames[3] = \"\"\n",
       "  frames[4] = \"\"\n",
       "  frames[5] = \"\"\n",
       "  frames[6] = \"\"\n",
       "  frames[7] = \"\"\n",
       "  frames[8] = \"\"\n",
       "  frames[9] = \"\"\n",
       "  frames[10] = \"\"\n",
       "  frames[11] = \"\"\n",
       "  frames[12] = \"\"\n",
       "  frames[13] = \"\"\n",
       "  frames[14] = \"\"\n",
       "  frames[15] = \"\"\n",
       "  frames[16] = \"\"\n",
       "  frames[17] = \"\"\n",
       "  frames[18] = \"\"\n",
       "  frames[19] = \"\"\n",
       "  frames[20] = \"\"\n",
       "  frames[21] = \"\"\n",
       "  frames[22] = \"\"\n",
       "  frames[23] = \"\"\n",
       "  frames[24] = \"\"\n",
       "  frames[25] = \"\"\n",
       "  frames[26] = \"\"\n",
       "  frames[27] = \"\"\n",
       "  frames[28] = \"\"\n",
       "  frames[29] = \"\"\n",
       "  frames[30] = \"\"\n",
       "  frames[31] = \"\"\n",
       "  frames[32] = \"\"\n",
       "  frames[33] = \"\"\n",
       "  frames[34] = \"\"\n",
       "  frames[35] = \"\"\n",
       "  frames[36] = \"\"\n",
       "  frames[37] = \"\"\n",
       "  frames[38] = \"\"\n",
       "  frames[39] = \"\"\n",
       "  frames[40] = \"\"\n",
       "  frames[41] = \"\"\n",
       "  frames[42] = \"\"\n",
       "  frames[43] = \"\"\n",
       "  frames[44] = \"\"\n",
       "  frames[45] = \"\"\n",
       "  frames[46] = \"\"\n",
       "  frames[47] = \"\"\n",
       "  frames[48] = \"\"\n",
       "  frames[49] = \"\"\n",
       "  frames[50] = \"\"\n",
       "  frames[51] = \"\"\n",
       "  frames[52] = \"\"\n",
       "  frames[53] = \"\"\n",
       "  frames[54] = \"\"\n",
       "  frames[55] = \"\"\n",
       "  frames[56] = \"\"\n",
       "  frames[57] = \"\"\n",
       "  frames[58] = \"\"\n",
       "  frames[59] = \"\"\n",
       "  frames[60] = \"\"\n",
       "  frames[61] = \"\"\n",
       "  frames[62] = \"\"\n",
       "  frames[63] = \"\"\n",
       "  frames[64] = \"\"\n",
       "  frames[65] = \"\"\n",
       "  frames[66] = \"\"\n",
       "  frames[67] = \"\"\n",
       "  frames[68] = \"\"\n",
       "  frames[69] = \"\"\n",
       "  frames[70] = \"\"\n",
       "  frames[71] = \"\"\n",
       "  frames[72] = \"\"\n",
       "  frames[73] = \"\"\n",
       "  frames[74] = \"\"\n",
       "  frames[75] = \"\"\n",
       "  frames[76] = \"\"\n",
       "  frames[77] = \"\"\n",
       "  frames[78] = \"\"\n",
       "  frames[79] = \"\"\n",
       "  frames[80] = \"\"\n",
       "  frames[81] = \"\"\n",
       "  frames[82] = \"\"\n",
       "  frames[83] = \"\"\n",
       "  frames[84] = \"\"\n",
       "  frames[85] = \"\"\n",
       "  frames[86] = \"\"\n",
       "  frames[87] = \"\"\n",
       "  frames[88] = \"\"\n",
       "  frames[89] = \"\"\n",
       "  frames[90] = \"\"\n",
       "  frames[91] = \"\"\n",
       "  frames[92] = \"\"\n",
       "  frames[93] = \"\"\n",
       "  frames[94] = \"\"\n",
       "  frames[95] = \"\"\n",
       "  frames[96] = \"\"\n",
       "  frames[97] = \"\"\n",
       "  frames[98] = \"\"\n",
       "  frames[99] = \"\"\n",
       "  frames[100] = \"\"\n",
       "  frames[101] = \"\"\n",
       "  frames[102] = \"\"\n",
       "  frames[103] = \"\"\n",
       "  frames[104] = \"\"\n",
       "  frames[105] = \"\"\n",
       "  frames[106] = \"\"\n",
       "  frames[107] = \"\"\n",
       "  frames[108] = \"\"\n",
       "  frames[109] = \"\"\n",
       "  frames[110] = \"\"\n",
       "  frames[111] = \"\"\n",
       "  frames[112] = \"\"\n",
       "  frames[113] = \"\"\n",
       "  frames[114] = \"\"\n",
       "  frames[115] = \"\"\n",
       "  frames[116] = \"\"\n",
       "  frames[117] = \"\"\n",
       "  frames[118] = \"\"\n",
       "  frames[119] = \"\"\n",
       "  frames[120] = \"\"\n",
       "  frames[121] = \"\"\n",
       "  frames[122] = \"\"\n",
       "  frames[123] = \"\"\n",
       "  frames[124] = \"\"\n",
       "  frames[125] = \"\"\n",
       "  frames[126] = \"\"\n",
       "  frames[127] = \"\"\n",
       "  frames[128] = \"\"\n",
       "  frames[129] = \"\"\n",
       "  frames[130] = \"\"\n",
       "  frames[131] = \"\"\n",
       "  frames[132] = \"\"\n",
       "  frames[133] = \"\"\n",
       "  frames[134] = \"\"\n",
       "  frames[135] = \"\"\n",
       "  frames[136] = \"\"\n",
       "  frames[137] = \"\"\n",
       "  frames[138] = \"\"\n",
       "  frames[139] = \"\"\n",
       "  frames[140] = \"\"\n",
       "  frames[141] = \"\"\n",
       "  frames[142] = \"\"\n",
       "  frames[143] = \"\"\n",
       "  frames[144] = \"\"\n",
       "  frames[145] = \"\"\n",
       "  frames[146] = \"\"\n",
       "  frames[147] = \"\"\n",
       "  frames[148] = \"\"\n",
       "  frames[149] = \"\"\n",
       "  frames[150] = \"\"\n",
       "  frames[151] = \"\"\n",
       "  frames[152] = \"\"\n",
       "  frames[153] = \"\"\n",
       "  frames[154] = \"\"\n",
       "  frames[155] = \"\"\n",
       "  frames[156] = \"\"\n",
       "  frames[157] = \"\"\n",
       "  frames[158] = \"\"\n",
       "  frames[159] = \"\"\n",
       "  frames[160] = \"\"\n",
       "  frames[161] = \"\"\n",
       "  frames[162] = \"\"\n",
       "  frames[163] = \"\"\n",
       "  frames[164] = \"\"\n",
       "  frames[165] = \"\"\n",
       "  frames[166] = \"\"\n",
       "  frames[167] = \"\"\n",
       "  frames[168] = \"\"\n",
       "  frames[169] = \"\"\n",
       "  frames[170] = \"\"\n",
       "  frames[171] = \"\"\n",
       "  frames[172] = \"\"\n",
       "  frames[173] = \"\"\n",
       "  frames[174] = \"\"\n",
       "  frames[175] = \"\"\n",
       "  frames[176] = \"\"\n",
       "  frames[177] = \"\"\n",
       "  frames[178] = \"\"\n",
       "  frames[179] = \"\"\n",
       "  frames[180] = \"\"\n",
       "  frames[181] = \"\"\n",
       "  frames[182] = \"\"\n",
       "  frames[183] = \"\"\n",
       "  frames[184] = \"\"\n",
       "  frames[185] = \"\"\n",
       "  frames[186] = \"\"\n",
       "  frames[187] = \"\"\n",
       "  frames[188] = \"\"\n",
       "  frames[189] = \"\"\n",
       "  frames[190] = \"\"\n",
       "  frames[191] = \"\"\n",
       "  frames[192] = \"\"\n",
       "  frames[193] = \"\"\n",
       "  frames[194] = \"\"\n",
       "  frames[195] = \"\"\n",
       "  frames[196] = \"\"\n",
       "  frames[197] = \"\"\n",
       "  frames[198] = \"\"\n",
       "  frames[199] = \"\"\n",
       "\n",
       "\n",
       "    /* set a timeout to make sure all the above elements are created before\n",
       "       the object is initialized. */\n",
       "    setTimeout(function() {\n",
       "        animTGHEMQVWAJRGKPBH = new Animation(frames, img_id, slider_id, 50, loop_select_id);\n",
       "    }, 0);\n",
       "  })()\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# main\n",
    "cartpole_env = Environment()\n",
    "cartpole_env.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
